cimport _common_

class Processor(_common_.Processor):
    cdef list _query(self, unicode line):
        return [Word(token, self._resolve(token)) for token in self._tokenise(line)]
        
    cdef list _tokenise(self, unicode line):
        """
        Not generic becuase, for example, parsing Japanese or Chinese phrases
        may involve searching clusters of kanji/hanzi from longest to shortest,
        which doesn't lend itself well to a linear parsing approach, while
        European-like languages have no such problems.
        """
        